package org.lilystudio.veryftp.util;

//import org.apache.lucene.*;
import java.util.*;
import java.io.*;
import org.apache.lucene.analysis.*;
public class AnalyzerUtil {
	public static Token[] tokensFromAnayzersis(Analyzer analyzer,String text)throws Exception{
		TokenStream stream =analyzer.tokenStream("contents", new StringReader(text));
		
		List<Token> tokenList=new ArrayList<Token>();
		
		while(true){
			Token token=stream.next();
			if(token==null) break;
			tokenList.add(token);
		}
		return (Token[])tokenList.toArray(new Token[0]);
	}
	public static void displayTokens(Analyzer analyzer,String text)throws Exception{
		System.out.println("Tokens as follows:");
		Token[] tokens=tokensFromAnayzersis(analyzer,text);
		
		for(Token token:tokens){
			System.out.println(token.termText());
		}
	}

}
